{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import json\n",
    "import csv\n",
    "import os\n",
    "import random\n",
    "from tempfile import gettempdir\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "import shutil\n",
    "\n",
    "# tf.enable_eager_execution()\n",
    "tf.executing_eagerly()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls -la ../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_loc = '../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy/ARC-Easy-Dev.jsonl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(file_loc) as f:\n",
    "    content = f.readlines()\n",
    "print(len(content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "content[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string = json.loads(content[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weird entries\n",
    "Some have numbered answers, fill in the blank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string = json.loads(content[8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write jsonl data to csv as labeled dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "regex = r\"(?<!\\d)[.,;:?](?!\\d)\"\n",
    "\n",
    "letter_answers = ['A', 'B', 'C', 'D', 'E']\n",
    "number_answers = ['1', '2', '3', '4', '5']\n",
    "\n",
    "def extract_question_answer(str_line):\n",
    "  \"\"\"Reads in line as string, returns tuple of question, answer\"\"\"\n",
    "  json_line = json.loads(str_line)\n",
    "  \n",
    "  question = json_line['question']['stem']\n",
    "  \n",
    "  choices = [choice['text'] for choice in json_line['question']['choices']]\n",
    "  \n",
    "  answer = json_line['answerKey']\n",
    "  if answer in letter_answers:\n",
    "    answer_idx = letter_answers.index(answer)\n",
    "  else: \n",
    "    answer_idx = number_answers.index(answer)\n",
    "  answer_text = json_line['question']['choices'][answer_idx]['text']\n",
    "  \n",
    "  question_formated = re.sub(\n",
    "    regex, \"\", question, 0).replace(\"\\\"\",\"\").replace(\"\\'\",\"\").replace(\",\",\"\").replace(\";\",\"\")\n",
    "  choices_formated = \";\".join(\n",
    "    [re.sub(\n",
    "      regex, \"\", choice, 0).replace(\"\\\"\",\"\").replace(\"\\'\",\"\").replace(\",\",\"\").replace(\";\",\"\") \n",
    "     for choice in choices])\n",
    "  \n",
    "  return question_formated, choices_formated, answer_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import codecs\n",
    "\n",
    "filename = \"train_data.csv\"\n",
    "file_loc = \"../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy/ARC-Easy-Train.jsonl\"\n",
    "\n",
    "if os.path.exists(filename):\n",
    "  os.remove(filename)\n",
    "  \n",
    "with io.open(file_loc, encoding='utf-8') as f:\n",
    "  with open(filename, \"w\") as out:\n",
    "    for line in f:\n",
    "      question, choices, answer_idx = extract_question_answer(line)\n",
    "      with codecs.open(filename, \"a\", \"utf-8\") as temp:\n",
    "        temp.write(question + \",\" + choices + \",\" + str(answer_idx) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import codecs\n",
    "\n",
    "filename = \"eval_data.csv\"\n",
    "file_loc = \"../maniac/arc/ARC-V1-Feb2018-2/ARC-Easy/ARC-Easy-Dev.jsonl\"\n",
    "\n",
    "if os.path.exists(filename):\n",
    "  os.remove(filename)\n",
    "  \n",
    "with io.open(file_loc, encoding='utf-8') as f:\n",
    "  with open(filename, \"w\") as out:\n",
    "    for line in f:\n",
    "      question, choices, answer_idx = extract_question_answer(line)\n",
    "      with codecs.open(filename, \"a\", \"utf-8\") as temp:\n",
    "        temp.write(question + \",\" + choices + \",\" + str(answer_idx) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -5 train_data.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -5 eval_data.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learn embeddings from ARC corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -10 ARC_Corpus.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "regex = r\"(?<!\\d)[.,;:?`\\\"\\'\\(\\)\\[\\]\\{\\}\\\\/“”](?!\\d)\"\n",
    "\n",
    "# Step 1: Download the data.\n",
    "filename = \"ARC_Corpus.txt\"\n",
    "\n",
    "# Read the data into a list of strings.\n",
    "def read_data(filename):\n",
    "  with open(filename, 'r') as myfile:\n",
    "    data = myfile.read().replace('\\n', ' ')\n",
    "    data = re.sub(\n",
    "      regex, \"\", data, 0).replace(\"(\",\" \").replace(\")\",\" \").replace(\"- \",\"\").replace(\"“\",\"\").replace(\"”\",\"\").replace(\"\\\"\",\"\").replace(\"\\'\",\"\").replace(\"  \",\" \")\n",
    "    data = data.split(' ')\n",
    "    data[:] = [word \n",
    "               for word in data if (word != \" \" and word != \"\\n\" and word != \"\")]\n",
    "    \n",
    "  return data\n",
    "\n",
    "vocabulary = read_data(filename)\n",
    "print(len(vocabulary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 2: Build the dictionary and replace rare words with UNK token.\n",
    "vocabulary_size = 50000\n",
    "\n",
    "def build_dataset(words, n_words):\n",
    "  \"\"\"Process raw inputs into a dataset.\"\"\"\n",
    "  count = [['UNK', -1]]\n",
    "  count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "  dictionary = dict()\n",
    "  for word, _ in count:\n",
    "    dictionary[word] = len(dictionary)\n",
    "  data = list()\n",
    "  unk_count = 0\n",
    "  for word in words:\n",
    "    index = dictionary.get(word, 0)\n",
    "    if index == 0:  # dictionary['UNK']\n",
    "      unk_count += 1\n",
    "    data.append(index)\n",
    "  count[0][1] = unk_count\n",
    "  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "  return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "# Filling 4 global variables:\n",
    "# data - list of codes (integers from 0 to vocabulary_size-1).\n",
    "#   This is the original text but words are replaced by their codes\n",
    "# count - map of words(strings) to count of occurrences\n",
    "# dictionary - map of words(strings) to their codes(integers)\n",
    "# reverse_dictionary - maps codes(integers) to words(strings)\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(vocabulary, vocabulary_size)\n",
    "del vocabulary  # Hint to reduce memory.\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])\n",
    "\n",
    "data_index = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 3: Function to generate a training batch for the skip-gram model.\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "  global data_index\n",
    "  assert batch_size % num_skips == 0\n",
    "  assert num_skips <= 2 * skip_window\n",
    "  batch = np.ndarray(\n",
    "    shape = (batch_size), \n",
    "    dtype = np.int32)\n",
    "  labels = np.ndarray(\n",
    "    shape = (batch_size, 1), \n",
    "    dtype = np.int32)\n",
    "  span = 2 * skip_window + 1  # [ skip_window target skip_window ]\n",
    "  buffer = collections.deque(maxlen=span)\n",
    "  if data_index + span > len(data):\n",
    "    data_index = 0\n",
    "  buffer.extend(data[data_index:data_index + span])\n",
    "  data_index += span\n",
    "  for i in range(batch_size // num_skips):\n",
    "    context_words = [w for w in range(span) if w != skip_window]\n",
    "    words_to_use = random.sample(context_words, num_skips)\n",
    "    for j, context_word in enumerate(words_to_use):\n",
    "      batch[i * num_skips + j] = buffer[skip_window]\n",
    "      labels[i * num_skips + j, 0] = buffer[context_word]\n",
    "    if data_index == len(data):\n",
    "      buffer[:] = data[:span]\n",
    "      data_index = span\n",
    "    else:\n",
    "      buffer.append(data[data_index])\n",
    "      data_index += 1\n",
    "  # Backtrack a little bit to avoid skipping words in the end of a batch\n",
    "  data_index = (data_index + len(data) - span) % len(data)\n",
    "  return batch, labels\n",
    "\n",
    "batch, labels = generate_batch(batch_size = 8, num_skips = 2, skip_window = 1)\n",
    "# This is first doing target word with left word, then target word with right word\n",
    "for i in range(8):\n",
    "  print(batch[i], reverse_dictionary[batch[i]],\n",
    "    '->', labels[i, 0], reverse_dictionary[labels[i, 0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 4: Build and train a skip-gram model.\n",
    "batch_size = 128\n",
    "embedding_size = 128  # Dimension of the embedding vector.\n",
    "skip_window = 1     # How many words to consider left and right.\n",
    "num_skips = 2     # How many times to reuse an input to generate a label.\n",
    "num_sampled = 64    # Number of negative examples to sample.\n",
    "\n",
    "# We pick a random validation set to sample nearest neighbors. Here we limit the\n",
    "# validation samples to the words that have a low numeric ID, which by\n",
    "# construction are also the most frequent. These 3 variables are used only for\n",
    "# displaying model accuracy, they don't affect calculation.\n",
    "valid_size = 16   # Random set of words to evaluate similarity on.\n",
    "valid_window = 100  # Only pick dev samples in the head of the distribution.\n",
    "# Generate a uniform random sample from np.arange(valid_window) of size valid_size without replacement\n",
    "valid_examples = np.random.choice(\n",
    "  a = valid_window, \n",
    "  size = valid_size, \n",
    "  replace = False)\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "  # Input data.\n",
    "  train_inputs = tf.placeholder(dtype = tf.int32, shape = [batch_size]) # batch_size vector\n",
    "  train_labels = tf.placeholder(dtype = tf.int32, shape = [batch_size, 1]) # batch_size x 1 matrix\n",
    "  valid_dataset = tf.constant(value = valid_examples, dtype = tf.int32) # valid_size vector\n",
    "  #print(\"valid_dataset = \", valid_dataset)\n",
    "\n",
    "  # Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "  with tf.device(\"/cpu:0\"):\n",
    "    # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "      initial_value = tf.random_uniform(\n",
    "        shape = [vocabulary_size, embedding_size], \n",
    "        minval = -1.0, \n",
    "        maxval = 1.0)) # embedding weights are vocabulary_size x embedding_size matrix\n",
    "    #print(\"embeddings = \", embeddings)\n",
    "    embed = tf.nn.embedding_lookup(\n",
    "      params = embeddings, \n",
    "      ids = train_inputs) # embedding_size x embedding_size matrix\n",
    "    #print(\"embed = \", embed)\n",
    "\n",
    "    # Construct the variables for the NCE loss\n",
    "    nce_weights = tf.Variable(\n",
    "      initial_value = tf.truncated_normal(\n",
    "        shape = [vocabulary_size, embedding_size], \n",
    "        stddev = 1.0 / math.sqrt(embedding_size))) # nce weights are vocabulary_size x embedding_size matrix\n",
    "    #print(\"nce_weights = \", nce_weights)\n",
    "    nce_biases = tf.Variable(\n",
    "      initial_value = tf.zeros(\n",
    "        shape = [vocabulary_size])) # nce biases are a vocabulary_size vector\n",
    "    #print(\"nce_biases = \", nce_biases)\n",
    "\n",
    "  # Compute the average NCE loss for the batch.\n",
    "  # tf.nce_loss automatically draws a new sample of the negative labels each\n",
    "  # time we evaluate the loss.\n",
    "  # Explanation of the meaning of NCE loss:\n",
    "  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.nce_loss(weights=nce_weights,\n",
    "           biases=nce_biases,\n",
    "           labels=train_labels,\n",
    "           inputs=embed,\n",
    "           num_sampled=num_sampled,\n",
    "           num_classes=vocabulary_size)) # scalar\n",
    "    #print(\"loss = \", loss)\n",
    "\n",
    "  # Construct the SGD optimizer using a learning rate of 1.0.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "\n",
    "  # Compute the cosine similarity between minibatch examples and all embeddings.\n",
    "  norm = tf.sqrt(\n",
    "    x = tf.reduce_sum(\n",
    "      input_tensor = tf.square(\n",
    "        x = embeddings), \n",
    "      axis = 1, \n",
    "      keepdims = True)) # vocabulary_size x 1 matrix, summed along the embedding_size axis\n",
    "  #print(\"norm = \", norm)\n",
    "  normalized_embeddings = embeddings / norm # vocabulary_size x embedding_size matrix\n",
    "  #print(\"normalized_embeddings = \", normalized_embeddings)\n",
    "  valid_embeddings = tf.nn.embedding_lookup(\n",
    "    params = normalized_embeddings, \n",
    "    ids = valid_dataset) # valid_size x embedding_size matrix\n",
    "  #print(\"valid_embeddings = \", valid_embeddings)\n",
    "  # cosine similarity\n",
    "  similarity = tf.matmul(\n",
    "    a = valid_embeddings, \n",
    "    b = normalized_embeddings, \n",
    "    transpose_b = True) #  valid_size x vocabulary_size = (valid_size x embedding_size matrix) x (vocabulary_size x embedding_size matrix)^T\n",
    "  #print(\"similarity = \", similarity)\n",
    "\n",
    "  # Add variable initializer.\n",
    "  init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 5: Begin training.\n",
    "num_steps = 100001\n",
    "\n",
    "with tf.Session(graph = graph) as session:\n",
    "  # We must initialize all variables before we use them.\n",
    "  init.run()\n",
    "  print(\"Initialized\")\n",
    "\n",
    "  average_loss = 0\n",
    "  for step in xrange(num_steps):\n",
    "    batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)\n",
    "    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "\n",
    "    # We perform one update step by evaluating the optimizer op (including it in the list of returned values for session.run()\n",
    "    _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "    average_loss += loss_val\n",
    "\n",
    "    if step % 2000 == 0:\n",
    "      if step > 0:\n",
    "        average_loss /= 2000\n",
    "      # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "      print(\"Average loss at step \", step, \": \", average_loss)\n",
    "      average_loss = 0\n",
    "\n",
    "    # Note that this is expensive (~20% slowdown if computed every 500 steps)\n",
    "    if step % 10000 == 0:\n",
    "      sim = similarity.eval()\n",
    "      for i in xrange(valid_size):\n",
    "        valid_word = reverse_dictionary[valid_examples[i]]\n",
    "        top_k = 8  # number of nearest neighbors\n",
    "        nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "        log_str = \"Nearest to %s:\" % valid_word # finding the k nearest neighbors to the validation word\n",
    "        for k in xrange(top_k):\n",
    "          close_word = reverse_dictionary[nearest[k]]\n",
    "          log_str = \"%s %s,\" % (log_str, close_word)\n",
    "        print(log_str)\n",
    "\n",
    "  # Get final embeddings after training is complete\n",
    "  final_embeddings = normalized_embeddings.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(\n",
    "  fname = \"vocab.tsv\", \n",
    "  X = np.array(object = list(dictionary.keys())), \n",
    "  fmt = '%s', \n",
    "  delimiter = '\\n') # write vocabulary to file\n",
    "np.savetxt(\n",
    "  fname = \"word_embeddings.csv\", \n",
    "  X = final_embeddings, \n",
    "  delimiter = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -10 vocab.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !head -5 word_embeddings.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read in trained embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pre_trained_embeddings_array = np.loadtxt(\n",
    "  fname = \"word_embeddings.csv\", \n",
    "  dtype = np.float64, \n",
    "  delimiter = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocabulary_size = 50000 & embedding_size = 128\n"
     ]
    }
   ],
   "source": [
    "vocabulary_size = pre_trained_embeddings_array.shape[0]\n",
    "embedding_size = pre_trained_embeddings_array.shape[1]\n",
    "print(\"vocabulary_size = {} & embedding_size = {}\".format(vocabulary_size, embedding_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create TensorFlow Continuous Bag of Words Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "arguments = {}\n",
    "# File arguments\n",
    "arguments[\"train_file_pattern\"] = \"train_data.csv\"\n",
    "arguments[\"eval_file_pattern\"] = \"eval_data.csv\"\n",
    "arguments[\"output_dir\"] = \"trained_model_cbow\"\n",
    "\n",
    "# Sequence shape hyperparameters\n",
    "arguments[\"batch_size\"] = 32\n",
    "\n",
    "# DNN hyperparameters\n",
    "arguments[\"dnn_hidden_units\"] = [1024, 256, 64]\n",
    "\n",
    "# Training parameters\n",
    "arguments[\"train_steps\"] = 10000\n",
    "arguments[\"learning_rate\"] = 0.1\n",
    "arguments[\"start_delay_secs\"] = 30\n",
    "arguments[\"throttle_secs\"] = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set logging to be level of INFO\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine CSV and label columns\n",
    "CSV_COLUMNS = \"question,choices,answer_idx\".split(',')\n",
    "LABEL_COLUMN = \"answer_idx\"\n",
    "VOCAB_FILE_PATH = \"vocab.tsv\" # where vocabulary is saved, dynamically set in train_and_eval function\n",
    "PADWORD = 'ZYXW'\n",
    "\n",
    "# Set default values for each CSV column\n",
    "DEFAULTS = [[\"\"], [\"\"], [0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an input function reading a file using the Dataset API\n",
    "# Then provide the results to the Estimator API\n",
    "def read_dataset(filename, mode, batch_size, params):\n",
    "  def _input_fn():\n",
    "    def decode_csv(value_column):\n",
    "      columns = tf.decode_csv(\n",
    "        records = value_column, record_defaults = DEFAULTS, field_delim = ',')\n",
    "      features = dict(zip(CSV_COLUMNS, columns))\n",
    "      labels = tf.cast(x = features.pop(LABEL_COLUMN), dtype = tf.int64)\n",
    "      \n",
    "      return features, labels\n",
    "    \n",
    "    # Create list of files that match pattern\n",
    "    file_list = tf.gfile.Glob(filename = filename)\n",
    "\n",
    "    # Create dataset from file list\n",
    "    dataset = tf.data.TextLineDataset(filenames = file_list)  # Read text file\n",
    "\n",
    "    # Decode the CSV file into a features dictionary of tensors\n",
    "    dataset = dataset.map(map_func = decode_csv)\n",
    "    \n",
    "    # Determine amount of times to repeat file based on if we are training or evaluating\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      num_epochs = None # indefinitely\n",
    "    else:\n",
    "      num_epochs = 1 # end-of-input after this\n",
    "\n",
    "    # Repeat files num_epoch times\n",
    "    dataset = dataset.repeat(count = num_epochs)\n",
    "\n",
    "    # Group the data into batches\n",
    "    dataset = dataset.batch(batch_size = batch_size)\n",
    "    \n",
    "    # Determine if we should shuffle based on if we are training or evaluating\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      dataset = dataset.shuffle(buffer_size = 10 * batch_size)\n",
    "\n",
    "    # Create a iterator and then pull the next batch of features from the example queue\n",
    "    batch_features, batch_labels = dataset.make_one_shot_iterator().get_next()\n",
    "\n",
    "    return batch_features, batch_labels\n",
    "  return _input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create our model function to be used in our custom estimator\n",
    "def cbow_text_question_answer(features, labels, mode, params):\n",
    "  # Function to split string tensors into multiple substring tensors\n",
    "  def split_strings(features):\n",
    "    # Function to split string tensors into substring tensors\n",
    "    def split_string_into_substrings(string_tensor, delimiter):\n",
    "      # Split string tensor into a sparse tensor based on delimiter\n",
    "      # indices: shape = (cur_batch_size, 2), values: shape = (cur_batch_size,), dense_shape: shape = (2,)\n",
    "      split_string_sparse_tensor = tf.string_split(\n",
    "        source = string_tensor, \n",
    "        delimiter = delimiter)\n",
    "\n",
    "      # Create a dense tensor of the float values that were converted from text csv\n",
    "      # shape = (cur_batch_size, max_substrings_across_batch)\n",
    "      split_string_dense_tensor = tf.sparse_tensor_to_dense(\n",
    "        sp_input = split_string_sparse_tensor, \n",
    "        default_value = PADWORD)\n",
    "\n",
    "      # The index for the sequence I am currently on, the first column of the sparse tensor indicies\n",
    "      # shape = (cur_batch_size,)\n",
    "      sequence_index = split_string_sparse_tensor.indices[:, 0]\n",
    "\n",
    "      # The index for the timestep I am currently on, the second column of the sparse tensor indicies\n",
    "      # shape = (cur_batch_size,)\n",
    "      time_index = split_string_sparse_tensor.indices[:, 1]\n",
    "\n",
    "      # The sequence lengths for each sequence\n",
    "      # shape = (cur_batch_size,)\n",
    "      lengths_vector = tf.segment_max(\n",
    "        data = time_index, \n",
    "        segment_ids = sequence_index) + 1\n",
    "\n",
    "      return split_string_dense_tensor, lengths_vector\n",
    "\n",
    "    # Function to split a single string of multiple setences into words\n",
    "    def split_multi_sentence_string_into_words(multi_sentence_string_tensor):\n",
    "      # split_multi_sentence_string_dense_tensor.shape = (cur_batch_size, max_num_choices_across_batch)\n",
    "      # num_sentences_vector.shape = (cur_batch_size,)\n",
    "      split_multi_sentence_string_dense_tensor, num_sentences_vector = \\\n",
    "        split_string_into_substrings(string_tensor = multi_sentence_string_tensor, delimiter = ';')\n",
    "\n",
    "      cur_batch_size = tf.shape(\n",
    "        input = split_multi_sentence_string_dense_tensor, \n",
    "        out_type = tf.int64)[0] # shape = ()\n",
    "\n",
    "      # Calculate the max number of choices across all questions in the batch\n",
    "      max_num_choices_across_batch = tf.reduce_max(\n",
    "        input_tensor = num_sentences_vector) # shape = ()\n",
    "\n",
    "      split_multi_sentence_string_dense_tensor_flattened = tf.reshape(\n",
    "        tensor = split_multi_sentence_string_dense_tensor, \n",
    "        shape = [cur_batch_size * max_num_choices_across_batch])\n",
    "\n",
    "      # split_multi_sentence_word_string_dense_tensor_flattened.shape = (cur_batch_size * max_num_choices_across_batch, max_num_words_across_batch)\n",
    "      # num_multi_sentence_word_tensor_flattened.shape = (cur_batch_size * max_num_choices_across_batch,)\n",
    "      split_multi_sentence_word_string_dense_tensor_flattened, num_multi_sentence_word_tensor_flattened = \\\n",
    "        split_string_into_substrings(string_tensor = split_multi_sentence_string_dense_tensor_flattened, delimiter = ' ')\n",
    "\n",
    "      max_num_words_across_batch = tf.reduce_max(\n",
    "        input_tensor = num_multi_sentence_word_tensor_flattened) # shape = ()\n",
    "\n",
    "      split_multi_sentence_word_string_dense_tensor = tf.reshape(\n",
    "        tensor = split_multi_sentence_word_string_dense_tensor_flattened, \n",
    "        shape = [cur_batch_size, max_num_choices_across_batch, max_num_words_across_batch])\n",
    "\n",
    "      num_multi_sentence_word_tensor = tf.reshape(\n",
    "        tensor = num_multi_sentence_word_tensor_flattened, \n",
    "        shape = [cur_batch_size, max_num_choices_across_batch])\n",
    "\n",
    "      return split_multi_sentence_word_string_dense_tensor, num_multi_sentence_word_tensor, split_multi_sentence_string_dense_tensor, num_sentences_vector\n",
    "\n",
    "    # Start split_strings function\n",
    "    # question_split_words_strings_tensor.shape = (cur_batch_size, max_num_question_words_across_batch)\n",
    "    # question_num_words.shape = (cur_batch_size,)\n",
    "    question_split_words_strings_tensor, question_num_words = \\\n",
    "      split_string_into_substrings(string_tensor = features[\"question\"], delimiter = ' ')\n",
    "\n",
    "    # choices_split_words_strings_tensor.shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch)\n",
    "    # choices_num_words.shape = (cur_batch_size, max_num_choices)\n",
    "    # choices_split_sentences_strings_tensor.shape = (cur_batch_size, max_num_choices)\n",
    "    # choices_num_sentences. shape = (cur_batch_size,)\n",
    "    choices_split_words_strings_tensor, choices_num_words, choices_split_sentences_strings_tensor, choices_num_sentences = \\\n",
    "      split_multi_sentence_string_into_words(features[\"choices\"])\n",
    "\n",
    "    return question_split_words_strings_tensor, question_num_words, choices_split_words_strings_tensor, choices_num_words, choices_split_sentences_strings_tensor, choices_num_sentences\n",
    "\n",
    "  # question_split_words_strings_tensor.shape = (cur_batch_size, max_num_question_words_across_batch)\n",
    "  # question_num_words.shape = (cur_batch_size,)\n",
    "  # choices_split_words_strings_tensor.shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch)\n",
    "  # choices_num_words.shape = (cur_batch_size, max_num_choices)\n",
    "  # choices_split_sentences_strings_tensor.shape = (cur_batch_size, max_num_choices)\n",
    "  # choices_num_sentences.shape = (cur_batch_size,)\n",
    "  question_split_words_strings_tensor, question_num_words, choices_split_words_strings_tensor, choices_num_words, choices_split_sentences_strings_tensor, choices_num_sentences = \\\n",
    "    split_strings(features)\n",
    "  \n",
    "  # Map each word to respective integer index\n",
    "  word_to_id_lookup_table = tf.contrib.lookup.index_table_from_file(\n",
    "    vocabulary_file = \"vocab.tsv\",\n",
    "    num_oov_buckets = 0,\n",
    "    vocab_size = vocabulary_size,\n",
    "    default_value = 0,  # for words not in vocabulary (OOV)\n",
    "    delimiter = ' ')\n",
    "\n",
    "  question_split_words_ids_tensor = word_to_id_lookup_table.lookup(\n",
    "    question_split_words_strings_tensor) # shape = (cur_batch_size, max_num_question_words_across_batch)\n",
    "  choices_split_words_ids_tensor = word_to_id_lookup_table.lookup(\n",
    "    choices_split_words_strings_tensor) # shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch)\n",
    "  \n",
    "  # Load trained embeddings into variable\n",
    "  embeddings_placeholder = tf.placeholder(\n",
    "    dtype = tf.float64, \n",
    "    shape = [vocabulary_size, embedding_size], \n",
    "    name = \"embedding_placeholder\") # shape = (vocabulary_size, embedding_size)\n",
    "\n",
    "  embeddings_variable = tf.Variable(\n",
    "    initial_value = embeddings_placeholder, \n",
    "    trainable = True, \n",
    "    name = \"embedding_variable\", \n",
    "    dtype = tf.float64, \n",
    "    expected_shape = [vocabulary_size, embedding_size]) # shape = (vocabulary_size, embedding_size)\n",
    "  \n",
    "  # Get dynamic batch size in case there was a partially filled batch\n",
    "  cur_batch_size = tf.shape(input = question_split_words_ids_tensor, out_type = tf.int64)[0] # shape = ()\n",
    "  \n",
    "  # Calculate the max number of choices across all questions in the batch\n",
    "  max_num_choices_across_batch = tf.reduce_max(input_tensor = choices_num_sentences) # shape = ()\n",
    "  \n",
    "  max_num_choice_words_across_batch = tf.reduce_max(input_tensor = choices_num_words) # shape = ()\n",
    "\n",
    "  # Gather the embedding vectors for the question's words\n",
    "  # shape = (cur_batch_size, max_num_question_words_across_batch, embedding_size)\n",
    "  question_split_words_embeddings_tensor = tf.nn.embedding_lookup(\n",
    "    params = embeddings_variable, \n",
    "    ids = question_split_words_ids_tensor)\n",
    "  \n",
    "  # Gather the embedding vectors for the answer's words\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch, max_num_choice_words_across_batch, embedding_size)\n",
    "  choices_split_words_embeddings_tensor = tf.nn.embedding_lookup(\n",
    "    params = embeddings_variable, \n",
    "    ids = choices_split_words_ids_tensor)\n",
    "  \n",
    "  # shape = (cur_batch_size, embedding_size)\n",
    "  question_cbow = tf.map_fn(\n",
    "    fn = lambda x: tf.reduce_mean(\n",
    "      input_tensor = tf.gather(\n",
    "        params = question_split_words_embeddings_tensor[x[0], :, :], \n",
    "        indices = tf.range(start = 0, limit = x[1], dtype = tf.int64),\n",
    "        axis = 0), \n",
    "      axis = 0), \n",
    "    elems = [tf.range(start = 0, limit = cur_batch_size, dtype = tf.int64), \n",
    "             question_num_words], \n",
    "    dtype = tf.float64)\n",
    "\n",
    "  ################################################################################\n",
    "  \n",
    "  # Create the input layer to our DNN\n",
    "  network = question_cbow # shape = shape = (cur_batch_size, embedding_size)\n",
    "  \n",
    "  # Add hidden layers with the given number of units/neurons per layer\n",
    "  for units in params['dnn_hidden_units']:\n",
    "    network = tf.layers.dense(\n",
    "      inputs = network, \n",
    "      units = units, \n",
    "      activation = tf.nn.relu) # shape = (cur_batch_size, dnn_hidden_units[i])\n",
    "\n",
    "  # Connect the final hidden layer to a dense layer with no activation to get the logits\n",
    "  logits = tf.layers.dense(\n",
    "    inputs = network, \n",
    "    units = embedding_size, \n",
    "    activation = None) # shape = (cur_batch_size, embedding_size)\n",
    "  print(\"cbow_text_question_answer: logits = \\n{}\".format(logits))\n",
    "\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch, max_num_choice_words_across_batch, embedding_size)\n",
    "  choices_split_words_embeddings_tensor_flattened = tf.reshape(\n",
    "    tensor = choices_split_words_embeddings_tensor, \n",
    "    shape = [cur_batch_size * max_num_choices_across_batch, max_num_choice_words_across_batch, embedding_size])\n",
    "\n",
    "  # shape = (cur_batch_size * max_num_choices_across_batch, embedding_size)\n",
    "  choices_flattened_cbow = tf.map_fn(\n",
    "    fn = lambda x: tf.reduce_mean(input_tensor = tf.gather(\n",
    "      params = choices_split_words_embeddings_tensor_flattened[x[0], :, :], \n",
    "      indices = tf.range(start = 0, limit = x[1], dtype = tf.int64), \n",
    "      axis = 0), axis = 0), \n",
    "    elems = [tf.range(start = 0, limit = cur_batch_size * max_num_choices_across_batch, dtype = tf.int64), \n",
    "             tf.reshape(tensor = choices_num_words, shape = [cur_batch_size * max_num_choices_across_batch])], \n",
    "    dtype = tf.float64)\n",
    "\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch, embedding_size)\n",
    "  choices_cbow = tf.reshape(\n",
    "    tensor = choices_flattened_cbow, \n",
    "    shape = [cur_batch_size, max_num_choices_across_batch, embedding_size])\n",
    "\n",
    "  # shape = (cur_batch_size, embedding_size)\n",
    "  logits_normalized = tf.nn.l2_normalize(x = logits, axis = 1)\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch, embedding_size)\n",
    "  choices_cbow_normalized = tf.nn.l2_normalize(choices_cbow, axis = 2)\n",
    "  # shape = (cur_batch_size, max_num_choices_across_batch)\n",
    "  cosine_similarities = tf.transpose(\n",
    "    a = tf.map_fn(\n",
    "      fn = lambda x: tf.reduce_sum(\n",
    "        input_tensor = tf.multiply(\n",
    "          x = logits_normalized, \n",
    "          y = choices_cbow_normalized[:, x, :]), \n",
    "        axis = 1), \n",
    "      elems = tf.range(start = 0, limit = max_num_choices_across_batch, dtype = tf.int64), \n",
    "      dtype = tf.float64))\n",
    "\n",
    "  # shape = (cur_batch_size,)\n",
    "  predicted_answer_index = tf.argmax(input = cosine_similarities, axis = 1)\n",
    "  \n",
    "  # 3. Loss function, training/eval ops\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    # shape = (cur_batch_size, embedding_size)\n",
    "    answer_cbow = tf.map_fn(\n",
    "      fn = lambda x: choices_cbow[x[0], x[1], :], \n",
    "      elems = [tf.range(start = 0, limit = cur_batch_size, dtype = tf.int64), labels], \n",
    "      dtype = tf.float64)\n",
    "    \n",
    "    loss = tf.losses.mean_squared_error(\n",
    "      labels = answer_cbow, predictions = logits)\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = params['learning_rate'],\n",
    "      optimizer = \"Adam\")\n",
    "    eval_metric_ops = {\n",
    "      \"rmse\": tf.metrics.root_mean_squared_error(\n",
    "        labels = answer_cbow, predictions = logits),\n",
    "      \"accuracy\": tf.metrics.accuracy(\n",
    "        labels = labels, predictions = predicted_answer_index)\n",
    "    }\n",
    "    \n",
    "    predictions_dict = None\n",
    "    export_outputs = None\n",
    "  else:\n",
    "    # shape = (cur_batch_size,)\n",
    "    predicted_answer_text = tf.gather(\n",
    "      params = choices_split_sentences_strings_tensor, \n",
    "      indices = predicted_answer_index, \n",
    "      axis = 1)\n",
    "  \n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    "\n",
    "    # 4. Create predictions\n",
    "    predictions_dict = {\n",
    "      \"cosine_similarities\": cosine_similarities, \n",
    "      \"predicted_answer_index\": predicted_answer_index, \n",
    "      \"question_text\": features[\"question\"],\n",
    "      \"predicted_answer_text\": predicted_answer_text}\n",
    "\n",
    "    # 5. Create export outputs\n",
    "    export_outputs = {\n",
    "      \"predict_export_outputs\": tf.estimator.export.PredictOutput(\n",
    "        outputs = predictions_dict)}\n",
    "\n",
    "  # 6. Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "  mode = mode,\n",
    "  predictions = predictions_dict,\n",
    "  loss = loss,\n",
    "  train_op = train_op,\n",
    "  eval_metric_ops = eval_metric_ops,\n",
    "  export_outputs = export_outputs,\n",
    "  scaffold = tf.train.Scaffold(init_feed_dict = {\n",
    "    embeddings_placeholder: pre_trained_embeddings_array}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create our serving input function to accept the data at serving and send it in the right format to our custom estimator\n",
    "def serving_input_fn():\n",
    "  # Create placeholders to accept the data sent to the model at serving time\n",
    "  # All features come in as a batch of strings, shape = (batch_size,)\n",
    "  # This was so because of passing the arrays to online ml-engine prediction\n",
    "  feature_placeholders = {\n",
    "    feature: tf.placeholder(\n",
    "      dtype = tf.string, \n",
    "      shape = [None]) \n",
    "    for feature in CSV_COLUMNS[0:-1]\n",
    "  }\n",
    "\n",
    "  # Create feature tensors\n",
    "  features = feature_placeholders\n",
    "\n",
    "  return tf.estimator.export.ServingInputReceiver(\n",
    "    features = features, \n",
    "    receiver_tensors = feature_placeholders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create estimator to train and evaluate\n",
    "def train_and_evaluate(args):\n",
    "  # Create our custome estimator using our model function\n",
    "  estimator = tf.estimator.Estimator(\n",
    "    model_fn = cbow_text_question_answer,\n",
    "    model_dir = args[\"output_dir\"],\n",
    "    params = {\n",
    "    \"batch_size\": args[\"batch_size\"], \n",
    "    \"dnn_hidden_units\": args[\"dnn_hidden_units\"], \n",
    "    \"learning_rate\": args[\"learning_rate\"]})\n",
    "  \n",
    "  # Create train spec to read in our training data\n",
    "  train_spec = tf.estimator.TrainSpec(\n",
    "    input_fn = read_dataset(\n",
    "      filename = args[\"train_file_pattern\"], \n",
    "      mode = tf.estimator.ModeKeys.TRAIN, \n",
    "      batch_size = args[\"batch_size\"],\n",
    "      params = args),\n",
    "    max_steps = args[\"train_steps\"])\n",
    "  \n",
    "  # Create exporter to save out the complete model to disk\n",
    "  exporter = tf.estimator.LatestExporter(\n",
    "    name = \"exporter\", \n",
    "    serving_input_receiver_fn = serving_input_fn)\n",
    "  \n",
    "  # Create eval spec to read in our validation data and export our model\n",
    "  eval_spec = tf.estimator.EvalSpec(\n",
    "    input_fn = read_dataset(\n",
    "      filename = args[\"eval_file_pattern\"], \n",
    "      mode = tf.estimator.ModeKeys.EVAL, \n",
    "      batch_size = args[\"batch_size\"],\n",
    "      params = args),\n",
    "    steps = None,\n",
    "    start_delay_secs = args[\"start_delay_secs\"], # start evaluating after N seconds\n",
    "    throttle_secs = args[\"throttle_secs\"],  # evaluate every N seconds\n",
    "    exporters = exporter)\n",
    "  \n",
    "  # Create train and evaluate loop to train and evaluate our estimator\n",
    "  tf.estimator.train_and_evaluate(\n",
    "    estimator = estimator, \n",
    "    train_spec = train_spec, \n",
    "    eval_spec = eval_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'trained_model_cbow', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0xb3e20c278>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "train_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "train\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./train_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.shuffle(buffer_size = 10 * batch_size) = \n",
      "<ShuffleDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "train\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "WARNING:tensorflow:From /Users/ryangillard/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/sparse_ops.py:1165: sparse_to_dense (from tensorflow.python.ops.sparse_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0xb41fd4710>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ryangillard/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into trained_model_cbow/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.003289172, step = 1\n",
      "INFO:tensorflow:global_step/sec: 3.10682\n",
      "INFO:tensorflow:loss = 0.029770348, step = 101 (32.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.51616\n",
      "INFO:tensorflow:loss = 0.01862548, step = 201 (28.440 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.76023\n",
      "INFO:tensorflow:loss = 0.010306964, step = 301 (26.594 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.71767\n",
      "INFO:tensorflow:loss = 0.009734049, step = 401 (26.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.76486\n",
      "INFO:tensorflow:loss = 0.024016531, step = 501 (26.561 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.57167\n",
      "INFO:tensorflow:loss = 0.0076896357, step = 601 (27.998 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.62561\n",
      "INFO:tensorflow:loss = 0.0091778925, step = 701 (27.581 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.65615\n",
      "INFO:tensorflow:loss = 0.013852488, step = 801 (27.351 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.74997\n",
      "INFO:tensorflow:loss = 0.01288187, step = 901 (26.667 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.67396\n",
      "INFO:tensorflow:loss = 0.010814713, step = 1001 (27.218 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.60265\n",
      "INFO:tensorflow:loss = 0.013119217, step = 1101 (27.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.73888\n",
      "INFO:tensorflow:loss = 0.04005226, step = 1201 (26.746 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.14845\n",
      "INFO:tensorflow:loss = 0.014794264, step = 1301 (31.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.32121\n",
      "INFO:tensorflow:loss = 0.016361497, step = 1401 (43.082 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.38371\n",
      "INFO:tensorflow:loss = 0.012413876, step = 1501 (41.954 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.39264\n",
      "INFO:tensorflow:loss = 0.006682898, step = 1601 (41.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.34738\n",
      "INFO:tensorflow:loss = 0.016291399, step = 1701 (42.601 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.36364\n",
      "INFO:tensorflow:loss = 0.005570389, step = 1801 (42.307 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1853 into trained_model_cbow/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "eval\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c68b3ccc0>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-20-20:48:03\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-1853\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-20-20:48:08\n",
      "INFO:tensorflow:Saving dict for global step 1853: accuracy = 0.27719298, global_step = 1853, loss = 0.01686296, rmse = 0.13013443\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1853: trained_model_cbow/model.ckpt-1853\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "None\n",
      "cbow_text_question_answer: mode = \n",
      "infer\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8ad871d0>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-1853\n",
      "WARNING:tensorflow:From /Users/ryangillard/anaconda3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py:1044: calling SavedModelBuilder.add_meta_graph_and_variables (from tensorflow.python.saved_model.builder_impl) with legacy_init_op is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Pass your op to the equivalent parameter main_op instead.\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model_cbow/export/exporter/temp-b'1550695688'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model_cbow/export/exporter/temp-b'1550695688'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.73756\n",
      "INFO:tensorflow:loss = 0.0082193855, step = 1901 (57.552 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2503\n",
      "INFO:tensorflow:loss = 0.014207803, step = 2001 (44.444 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.2162\n",
      "INFO:tensorflow:loss = 0.004952169, step = 2101 (45.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.25086\n",
      "INFO:tensorflow:loss = 0.006236677, step = 2201 (44.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.26734\n",
      "INFO:tensorflow:loss = 0.0025089958, step = 2301 (44.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.24318\n",
      "INFO:tensorflow:loss = 0.015314674, step = 2401 (44.573 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.23332\n",
      "INFO:tensorflow:loss = 0.009655356, step = 2501 (44.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.27901\n",
      "INFO:tensorflow:loss = 0.01057509, step = 2601 (43.881 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.50687\n",
      "INFO:tensorflow:loss = 0.01230526, step = 2701 (39.888 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.12195\n",
      "INFO:tensorflow:loss = 0.0063853753, step = 2801 (47.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.23966\n",
      "INFO:tensorflow:loss = 0.017567394, step = 2901 (44.656 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.24844\n",
      "INFO:tensorflow:loss = 0.006753451, step = 3001 (44.470 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.29557\n",
      "INFO:tensorflow:loss = 0.010895498, step = 3101 (43.562 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3180 into trained_model_cbow/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "eval\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8b3ad940>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-20-20:58:03\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-3180\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-20-20:58:07\n",
      "INFO:tensorflow:Saving dict for global step 3180: accuracy = 0.28070176, global_step = 3180, loss = 0.018433724, rmse = 0.13603885\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 3180: trained_model_cbow/model.ckpt-3180\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "None\n",
      "cbow_text_question_answer: mode = \n",
      "infer\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8b07e470>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-3180\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model_cbow/export/exporter/temp-b'1550696287'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model_cbow/export/exporter/temp-b'1550696287'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.86126\n",
      "INFO:tensorflow:loss = 0.006005275, step = 3201 (53.727 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.31141\n",
      "INFO:tensorflow:loss = 0.016176566, step = 3301 (43.264 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.32928\n",
      "INFO:tensorflow:loss = 0.008967006, step = 3401 (42.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.00274\n",
      "INFO:tensorflow:loss = 0.014916587, step = 3501 (33.303 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.5172\n",
      "INFO:tensorflow:loss = 0.008902075, step = 3601 (28.431 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.70284\n",
      "INFO:tensorflow:loss = 0.00873332, step = 3701 (27.006 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.41601\n",
      "INFO:tensorflow:loss = 0.016656753, step = 3801 (29.274 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.29584\n",
      "INFO:tensorflow:loss = 0.010985583, step = 3901 (30.341 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.82748\n",
      "INFO:tensorflow:loss = 0.015956512, step = 4001 (35.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.28475\n",
      "INFO:tensorflow:loss = 0.005795887, step = 4101 (43.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.5563\n",
      "INFO:tensorflow:loss = 0.007544564, step = 4201 (39.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.46527\n",
      "INFO:tensorflow:loss = 0.008531812, step = 4301 (40.563 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.40954\n",
      "INFO:tensorflow:loss = 0.0067192633, step = 4401 (41.502 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.57198\n",
      "INFO:tensorflow:loss = 0.024930127, step = 4501 (38.881 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.33927\n",
      "INFO:tensorflow:loss = 0.01780536, step = 4601 (42.754 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.59845\n",
      "INFO:tensorflow:loss = 0.0054628695, step = 4701 (38.478 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 4782 into trained_model_cbow/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "eval\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c4f690470>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-20-21:08:00\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-4782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-20-21:08:02\n",
      "INFO:tensorflow:Saving dict for global step 4782: accuracy = 0.25789472, global_step = 4782, loss = 0.01606264, rmse = 0.12702988\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 4782: trained_model_cbow/model.ckpt-4782\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "None\n",
      "cbow_text_question_answer: mode = \n",
      "infer\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8a4744a8>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-4782\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model_cbow/export/exporter/temp-b'1550696882'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model_cbow/export/exporter/temp-b'1550696882'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 2.78553\n",
      "INFO:tensorflow:loss = 0.010078195, step = 4801 (35.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.47473\n",
      "INFO:tensorflow:loss = 0.0073365048, step = 4901 (40.408 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.65327\n",
      "INFO:tensorflow:loss = 0.010442979, step = 5001 (37.696 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.28038\n",
      "INFO:tensorflow:loss = 0.0073194336, step = 5101 (43.846 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.5701\n",
      "INFO:tensorflow:loss = 0.010263212, step = 5201 (38.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.40458\n",
      "INFO:tensorflow:loss = 0.008348593, step = 5301 (41.595 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.60501\n",
      "INFO:tensorflow:loss = 0.010112425, step = 5401 (38.379 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.44741\n",
      "INFO:tensorflow:loss = 0.0019301258, step = 5501 (40.860 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.46618\n",
      "INFO:tensorflow:loss = 0.013123644, step = 5601 (40.549 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.57589\n",
      "INFO:tensorflow:loss = 0.011055142, step = 5701 (38.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.36229\n",
      "INFO:tensorflow:loss = 0.0069514527, step = 5801 (42.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.60763\n",
      "INFO:tensorflow:loss = 0.013989529, step = 5901 (38.349 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.35444\n",
      "INFO:tensorflow:loss = 0.008943095, step = 6001 (42.473 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.57602\n",
      "INFO:tensorflow:loss = 0.0105953105, step = 6101 (38.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.2435\n",
      "INFO:tensorflow:loss = 0.009921187, step = 6201 (30.831 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.10639\n",
      "INFO:tensorflow:loss = 0.0068116398, step = 6301 (32.192 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 6309 into trained_model_cbow/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "eval\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8a676e48>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-20-21:18:03\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-6309\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-20-21:18:06\n",
      "INFO:tensorflow:Saving dict for global step 6309: accuracy = 0.2736842, global_step = 6309, loss = 0.016208958, rmse = 0.12743159\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 6309: trained_model_cbow/model.ckpt-6309\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "None\n",
      "cbow_text_question_answer: mode = \n",
      "infer\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c68dc7470>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-6309\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model_cbow/export/exporter/temp-b'1550697486'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model_cbow/export/exporter/temp-b'1550697486'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.99707\n",
      "INFO:tensorflow:loss = 0.0038743427, step = 6401 (50.079 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.43317\n",
      "INFO:tensorflow:loss = 0.006322906, step = 6501 (41.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.60167\n",
      "INFO:tensorflow:loss = 0.036513668, step = 6601 (38.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.33697\n",
      "INFO:tensorflow:loss = 0.006726485, step = 6701 (42.794 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.99989\n",
      "INFO:tensorflow:loss = 0.015636843, step = 6801 (33.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.17177\n",
      "INFO:tensorflow:loss = 0.014833757, step = 6901 (31.529 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.29868\n",
      "INFO:tensorflow:loss = 0.012105912, step = 7001 (43.503 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.62054\n",
      "INFO:tensorflow:loss = 0.0104391575, step = 7101 (38.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.35272\n",
      "INFO:tensorflow:loss = 0.006902692, step = 7201 (42.498 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.53995\n",
      "INFO:tensorflow:loss = 0.0075269816, step = 7301 (39.371 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.35462\n",
      "INFO:tensorflow:loss = 0.006943245, step = 7401 (42.475 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.11697\n",
      "INFO:tensorflow:loss = 0.006939531, step = 7501 (32.077 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.64401\n",
      "INFO:tensorflow:loss = 0.013063605, step = 7601 (27.442 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.79858\n",
      "INFO:tensorflow:loss = 0.048640646, step = 7701 (26.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.80227\n",
      "INFO:tensorflow:loss = 0.0039394936, step = 7801 (26.300 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.57309\n",
      "INFO:tensorflow:loss = 0.0065660286, step = 7901 (27.987 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7962 into trained_model_cbow/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "eval\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8aa895c0>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-20-21:28:03\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-7962\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-20-21:28:07\n",
      "INFO:tensorflow:Saving dict for global step 7962: accuracy = 0.25263157, global_step = 7962, loss = 0.014908546, rmse = 0.12232206\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 7962: trained_model_cbow/model.ckpt-7962\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "None\n",
      "cbow_text_question_answer: mode = \n",
      "infer\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c4f74b470>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-7962\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model_cbow/export/exporter/temp-b'1550698087'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model_cbow/export/exporter/temp-b'1550698087'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 2.09938\n",
      "INFO:tensorflow:loss = 0.0071006697, step = 8001 (47.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.47548\n",
      "INFO:tensorflow:loss = 0.008226944, step = 8101 (40.396 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.18596\n",
      "INFO:tensorflow:loss = 0.0073178173, step = 8201 (45.753 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.5758\n",
      "INFO:tensorflow:loss = 0.008302448, step = 8301 (38.817 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.26937\n",
      "INFO:tensorflow:loss = 0.022928549, step = 8401 (44.065 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.53895\n",
      "INFO:tensorflow:loss = 0.008535417, step = 8501 (39.392 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.38003\n",
      "INFO:tensorflow:loss = 0.009577753, step = 8601 (42.010 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.51409\n",
      "INFO:tensorflow:loss = 0.012811612, step = 8701 (39.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.49957\n",
      "INFO:tensorflow:loss = 0.007964098, step = 8801 (40.013 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.35662\n",
      "INFO:tensorflow:loss = 0.015844341, step = 8901 (42.428 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.57916\n",
      "INFO:tensorflow:loss = 0.018224217, step = 9001 (38.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.31883\n",
      "INFO:tensorflow:loss = 0.013316942, step = 9101 (43.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.5468\n",
      "INFO:tensorflow:loss = 0.008785208, step = 9201 (39.262 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.34428\n",
      "INFO:tensorflow:loss = 0.014283315, step = 9301 (42.658 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 9391 into trained_model_cbow/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "eval\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8a2333c8>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-20-21:38:04\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-9391\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-20-21:38:08\n",
      "INFO:tensorflow:Saving dict for global step 9391: accuracy = 0.28947368, global_step = 9391, loss = 0.012549762, rmse = 0.11212715\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 9391: trained_model_cbow/model.ckpt-9391\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "None\n",
      "cbow_text_question_answer: mode = \n",
      "infer\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c68e8b550>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-9391\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model_cbow/export/exporter/temp-b'1550698688'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model_cbow/export/exporter/temp-b'1550698688'/saved_model.pb\n",
      "INFO:tensorflow:global_step/sec: 1.90877\n",
      "INFO:tensorflow:loss = 0.0087493, step = 9401 (52.389 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.45202\n",
      "INFO:tensorflow:loss = 0.0038901668, step = 9501 (40.783 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.3565\n",
      "INFO:tensorflow:loss = 0.005242783, step = 9601 (42.436 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.59759\n",
      "INFO:tensorflow:loss = 0.009196525, step = 9701 (38.504 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.35339\n",
      "INFO:tensorflow:loss = 0.0045679607, step = 9801 (42.485 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.55153\n",
      "INFO:tensorflow:loss = 0.006922136, step = 9901 (39.192 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10000 into trained_model_cbow/model.ckpt.\n",
      "\n",
      "read_dataset: _input_fn: filename = \n",
      "eval_data.csv\n",
      "read_dataset: _input_fn: mode = \n",
      "eval\n",
      "read_dataset: _input_fn: batch_size = \n",
      "32\n",
      "read_dataset: _input_fn: params = \n",
      "{'train_file_pattern': 'train_data.csv', 'eval_file_pattern': 'eval_data.csv', 'output_dir': 'trained_model_cbow', 'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'train_steps': 10000, 'learning_rate': 0.1, 'start_delay_secs': 30, 'throttle_secs': 30}\n",
      "\n",
      "\n",
      "read_dataset: _input_fn: file_list = \n",
      "['./eval_data.csv']\n",
      "read_dataset: _input_fn: dataset.TextLineDataset(file_list) = \n",
      "<TextLineDataset shapes: (), types: tf.string>\n",
      "\n",
      "read_dataset: _input_fn: decode_csv: value_column = \n",
      "Tensor(\"arg0:0\", shape=(), dtype=string, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: decode_csv: columns = \n",
      "[<tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>]\n",
      "read_dataset: _input_fn: decode_csv: features = \n",
      "{'question': <tf.Tensor 'DecodeCSV:0' shape=() dtype=string>, 'choices': <tf.Tensor 'DecodeCSV:1' shape=() dtype=string>, 'answer_idx': <tf.Tensor 'DecodeCSV:2' shape=() dtype=int32>}\n",
      "read_dataset: _input_fn: decode_csv: labels = \n",
      "Tensor(\"Cast:0\", shape=(), dtype=int64, device=/device:CPU:0)\n",
      "read_dataset: _input_fn: dataset.map(decode_csv) = \n",
      "<MapDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.repeat(num_epochs) = \n",
      "<RepeatDataset shapes: ({question: (), choices: ()}, ()), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: dataset.batch(batch_size) = \n",
      "<BatchDataset shapes: ({question: (?,), choices: (?,)}, (?,)), types: ({question: tf.string, choices: tf.string}, tf.int64)>\n",
      "read_dataset: _input_fn: batch_features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "read_dataset: _input_fn: batch_labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "Tensor(\"IteratorGetNext:2\", shape=(?,), dtype=int64, device=/device:CPU:0)\n",
      "cbow_text_question_answer: mode = \n",
      "eval\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'IteratorGetNext:1' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:1\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"IteratorGetNext:0\", shape=(?,), dtype=string, device=/device:CPU:0)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8a125780>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: answer_cbow = \n",
      "Tensor(\"map_3/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-02-20-21:42:21\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-10000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-02-20-21:42:24\n",
      "INFO:tensorflow:Saving dict for global step 10000: accuracy = 0.24561404, global_step = 10000, loss = 0.01294617, rmse = 0.11393472\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 10000: trained_model_cbow/model.ckpt-10000\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: features = {'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "cbow_text_question_answer: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "cbow_text_question_answer: labels = \n",
      "None\n",
      "cbow_text_question_answer: mode = \n",
      "infer\n",
      "cbow_text_question_answer: params = \n",
      "{'batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'learning_rate': 0.1}\n",
      "\n",
      "cbow_text_question_answer: split_strings: features = \n",
      "{'question': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'choices': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Placeholder:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_string_tensor = \n",
      "Tensor(\"Placeholder_1:0\", shape=(?,), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_1:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_1:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_1:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_sequence_index = \n",
      "Tensor(\"strided_slice_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: multiple_sentence_time_index = \n",
      "Tensor(\"strided_slice_3:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_sentences: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_sentences_vector = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: current_batch_size = \n",
      "Tensor(\"strided_slice_4:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_string_dense_tensor_flattened = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_string_tensor = \n",
      "Tensor(\"Reshape:0\", shape=(?,), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_sparse_tensor = \n",
      "SparseTensor(indices=Tensor(\"StringSplit_2:0\", shape=(?, 2), dtype=int64), values=Tensor(\"StringSplit_2:1\", shape=(?,), dtype=string), dense_shape=Tensor(\"StringSplit_2:2\", shape=(2,), dtype=int64))\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: split_sentence_string_dense_tensor = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_sequence_index = \n",
      "Tensor(\"strided_slice_5:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_time_index = \n",
      "Tensor(\"strided_slice_6:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_sentence_answer: split_strings: split_sentence_string_into_words: sentence_lengths_vector = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor_flattened = \n",
      "Tensor(\"SparseToDense_2:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor_flattened = \n",
      "Tensor(\"add_2:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: max_number_of_words_across_batch = \n",
      "Tensor(\"Max_1:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: split_multiple_sentence_word_string_dense_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: split_multiple_sentence_string_into_words: number_of_multiple_sentence_word_tensor = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: split_strings: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: split_strings: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "\n",
      "cbow_text_question_answer: split_strings: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "\n",
      "cbow_text_question_answer: question_split_words_strings_tensor = \n",
      "Tensor(\"SparseToDense:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: question_number_of_words = \n",
      "Tensor(\"add:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_strings_tensor = \n",
      "Tensor(\"Reshape_1:0\", shape=(?, ?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_words = \n",
      "Tensor(\"Reshape_2:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_sentences_strings_tensor = \n",
      "Tensor(\"SparseToDense_1:0\", shape=(?, ?), dtype=string)\n",
      "cbow_text_question_answer: choices_number_of_sentences = \n",
      "Tensor(\"add_1:0\", shape=(?,), dtype=int64)\n",
      "cbow_text_question_answer: word_to_id_lookup_table = \n",
      "<tensorflow.python.ops.lookup_ops.HashTable object at 0x1c8a238438>\n",
      "cbow_text_question_answer: question_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup:0\", shape=(?, ?), dtype=int64)\n",
      "cbow_text_question_answer: choices_split_words_ids_tensor = \n",
      "Tensor(\"hash_table_Lookup_1:0\", shape=(?, ?, ?), dtype=int64)\n",
      "cbow_text_question_answer: embeddings_placeholder = \n",
      "Tensor(\"embedding_placeholder:0\", shape=(50000, 128), dtype=float64)\n",
      "cbow_text_question_answer: embeddings_variable = \n",
      "<tf.Variable 'embedding_variable:0' shape=(50000, 128) dtype=float64_ref>\n",
      "cbow_text_question_answer: current_batch_size = \n",
      "Tensor(\"strided_slice_7:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choices_across_batch = \n",
      "Tensor(\"Max_2:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: max_number_of_choice_words_across_batch = \n",
      "Tensor(\"Max_3:0\", shape=(), dtype=int64)\n",
      "cbow_text_question_answer: question_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup/Identity:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor = \n",
      "Tensor(\"embedding_lookup_1/Identity:0\", shape=(?, ?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: question_cbow = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = \n",
      "Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float64), units = 1024\n",
      "cbow_text_question_answer: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float64), units = 256\n",
      "cbow_text_question_answer: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float64), units = 64\n",
      "cbow_text_question_answer: logits = \n",
      "Tensor(\"dense_3/BiasAdd:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_split_words_embeddings_tensor_flattened = \n",
      "Tensor(\"Reshape_3:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_flattened_cbow = \n",
      "Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow = \n",
      "Tensor(\"Reshape_5:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: logits_normalized = \n",
      "Tensor(\"l2_normalize:0\", shape=(?, 128), dtype=float64)\n",
      "cbow_text_question_answer: choices_cbow_normalized = \n",
      "Tensor(\"l2_normalize_1:0\", shape=(?, ?, 128), dtype=float64)\n",
      "cbow_text_question_answer: cosine_similarities = \n",
      "Tensor(\"transpose:0\", shape=(?, ?), dtype=float64)\n",
      "cbow_text_question_answer: predicted_answer_index = \n",
      "Tensor(\"ArgMax:0\", shape=(?,), dtype=int64)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from trained_model_cbow/model.ckpt-10000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:Assets written to: trained_model_cbow/export/exporter/temp-b'1550698944'/assets\n",
      "INFO:tensorflow:SavedModel written to: trained_model_cbow/export/exporter/temp-b'1550698944'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.0071343062.\n"
     ]
    }
   ],
   "source": [
    "# Run the model\n",
    "shutil.rmtree(arguments[\"output_dir\"], ignore_errors = True) # start fresh each time\n",
    "train_and_evaluate(arguments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
